# -*- coding: utf-8 -*-
import scrapy

import random
import logging
import json
import re
from copy import copy, deepcopy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem
from scrapy.selector import Selector

logger = logging.getLogger(__name__)


class LuoyangSpider(scrapy.Spider):
    name = 'luoyang'
    allowed_domains = ['lyzbj.org.cn', '123.7.180.231']
    start_urls = ['http://www.lyzbj.org.cn/showysz?1=1']
    permit_li_url = 'http://www.lyzbj.org.cn/showysz?xm=&zl=&ysz=&pageindex={}'  # GET 预售许可证列表url
    room_li_url = 'http://123.7.180.231:810/tt/business/buildingRooms_xml.asp?functiontype=6&sid={}&{}'  # POST 房号列表url

    def parse(self, response):
        """
        预售许可证列表页
        :param response:
        :return:
        """
        # 获取当前页数
        try:
            page_num = copy(response.meta['page_num'])
        except:
            page_num = 1
        # 获取预售许可证分组
        permit_tr_li = response.xpath("//table[@class='fl-table']/tbody//tr")
        if permit_tr_li:
            filter_set = set()
            for permit_tr in permit_tr_li:
                item_eg = FdcEstateGuidItem()
                item_eg['projectName'] = permit_tr.xpath("./td[2]//text()").extract_first()  # 项目名称
                item_eg['projectAddress'] = permit_tr.xpath("./td[3]//text()").extract_first()  # 项目地址
                item_eg['preSalePermit'] = permit_tr.xpath("./td[4]//text()").extract_first()  # 预售许可证
                item_eg['approvalDate'] = permit_tr.xpath("./td[5]//text()").extract_first()  # 批准日期
                # 利用集合特性，进行预售许可证信息去重
                before_len = len(filter_set)
                filter_set.add(json.dumps(dict(item_eg)))
                after_len = len(filter_set)
                if before_len != after_len:
                    item_eg['permitId'] = permit_tr.xpath("./td[1]//text()").extract_first()  # 许可证id
                    permit_url = permit_tr.xpath("./td[2]//@href").extract_first()  # GET 预售许可证详情url
                    # 获取预售许可证详情
                    if permit_url:
                        yield response.follow(
                            permit_url,
                            callback=self.parse_permit_detail,
                            meta=dict(item_eg=deepcopy(item_eg), ),
                            priority=20,
                        )
                    else:
                        logger.error('第{}页-{}-{} 预售许可证url获取为空'.format(page_num, item_eg['projectName'],
                                                                      item_eg['preSalePermit']))
            del filter_set
            del permit_tr_li
        else:
            logger.error(f'第{page_num}页 预售许可证列表获取为空')

        # 翻页
        total_num_str = response.xpath("//div[@class='pglist']/div/a[1]/text()").extract_first()
        total_page_num = self.get_total_page_num(total_num_str)
        for page_num in range(2, total_page_num + 1):
            yield scrapy.Request(
                self.permit_li_url.format(page_num),
                meta=dict(page_num=deepcopy(page_num), ),
                dont_filter=False,
                priority=10,
            )

    def parse_permit_detail(self, response):
        """
        获取预售许可证详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        # 提取项目信息
        item_eg['developerName'] = response.xpath(
            "//td[contains(text(),'开发企业')]/following-sibling::td[1]/text()").extract_first()  # 开发商
        item_eg['preSaleArea'] = response.xpath(
            "//td[contains(text(),'楼栋面积')]/following-sibling::td[1]/text()").extract_first()  # 预售面积
        item_eg['landPermit'] = response.xpath(
            "//td[contains(text(),'土地使用权证')]/following-sibling::td[1]/text()").extract_first()  # 土地许可证
        item_eg['planningPermit'] = response.xpath(
            "//td[contains(text(),'规划许可证')]/following-sibling::td[1]/text()").extract_first()  # 规划许可证
        item_eg['constructionPermit'] = response.xpath(
            "//td[contains(text(),'施工许可证')]/following-sibling::td[1]/text()").extract_first()  # 施工许可证
        item_eg['landStartingDate'] = response.xpath(
            "//td[contains(text(),'土地起始日期')]/following-sibling::td[1]/text()").extract_first()  # 土地起始日期
        item_eg['landTerminalDate'] = response.xpath(
            "//td[contains(text(),'土地结束日期')]/following-sibling::td[1]/text()").extract_first()  # 土地终止日期
        item_eg['preSaleDepositBank'] = response.xpath(
            "//td[contains(text(),'监管银行')]/following-sibling::td[1]/text()").extract_first()  # 监管银行
        item_eg['preSaleAccountNum'] = response.xpath(
            "//td[contains(text(),'监管账号')]/following-sibling::td[1]/text()").extract_first()  # 监管银行
        yield item_eg

        # 提取楼栋列表分组
        building_tr_li = response.xpath("//div[contains(text(),'楼栋信息')]/following-sibling::table/tbody/tr")
        if building_tr_li:
            for building_tr in building_tr_li:
                item_bd = FdcBuildingItem()
                item_bd['projectName'] = item_eg['projectName']  # 项目名称
                item_bd['preSalePermit'] = item_eg['preSalePermit']  # 预售许可证
                item_bd['permitId'] = item_eg['permitId']  # 许可证id
                item_bd['buildingId'] = building_tr.xpath("./td[1]//text()").extract_first()  # 楼栋id
                item_bd['blockName'] = building_tr.xpath("./td[2]//text()").extract_first()  # 楼栋名字
                item_bd['totalRoomNo'] = building_tr.xpath("./td[3]//text()").extract_first()  # 总套数
                item_bd['buildingTotalArea'] = building_tr.xpath("./td[4]//text()").extract_first()  # 总建筑面积
                item_bd['residenceProposedPrice'] = building_tr.xpath("./td[5]//text()").extract_first()  # 住宅拟售价格
                item_bd['saleableRoomNo'] = building_tr.xpath("./td[6]//text()").extract_first()  # 可售套数

                room_li_params_url = building_tr.xpath("./td[7]/a[1]/@href").extract_first()  # GET 房号列表参数url
                # 获取房号列表url参数
                yield scrapy.Request(
                    room_li_params_url,
                    callback=self.parse_room_li_params,
                    meta=dict(item_bd=deepcopy(item_bd), ),
                    priority=30,
                )
        else:
            logger.warning(
                '{} {}-{} 楼栋列表获取为空'.format(response.request.url, item_eg['projectName'], item_eg['preSalePermit']))

    def parse_room_li_params(self, response):
        """
        获取房号列表url参数
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        try:
            # 构造房号列表url
            room_li_url = self.create_room_li_url(response.body.decode('gb2312'))
            assert room_li_url
        except Exception as e:
            logger.error('{}-{}-{} 房号列表url构造失败，error_msg:{}'.format(item_bd['projectName'], item_bd['preSalePermit'],
                                                                    item_bd['blockName'], e))
        else:
            # 获取房号列表
            yield scrapy.Request(
                room_li_url,
                method='POST',
                callback=self.parse_room_li,
                meta=dict(item_bd=deepcopy(item_bd), ),
                priority=40
            )

    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        xml_str = response.body.decode('gb2312')
        # xml字符串处理
        xml_obj = Selector(text=xml_str, type='xml')
        bd_unit_set = set()  # 用于楼栋单元统计
        bd_floor_set = set()  # 用于楼栋总楼层统计
        # 房号列表
        room_row_li = xml_obj.xpath("//row")
        # 遍历提取房号信息
        for room_row in room_row_li:
            item_rm = FdcRoomItem()
            item_rm['projectName'] = item_bd['projectName']
            item_rm['preSalePermit'] = item_bd['preSalePermit']
            item_rm['permitId'] = item_bd['permitId']
            item_rm['blockName'] = item_bd['blockName']
            item_rm['buildingId'] = item_bd['buildingId']
            item_rm['roomNo'] = room_row.xpath("./PART/text()").extract_first()  # 房号
            item_rm['roomId'] = room_row.xpath("./BLDROOMID/text()").extract_first()  # 房屋id
            item_rm['buildingArea'] = room_row.xpath("./BUILDAREA/text()").extract_first()  # 建筑面积
            item_rm['innerArea'] = room_row.xpath("./SETAREA/text()").extract_first()  # 套内面积
            item_rm['apportionmentArea'] = room_row.xpath("./PUBLICAREA/text()").extract_first()  # 分摊面积
            item_rm['planningUse'] = room_row.xpath("./DESIGNUSAGE/text()").extract_first()  # 规划用途
            item_rm['roomUse'] = room_row.xpath("./USAGE/text()").extract_first()  # 房屋用途
            item_rm['roomStructure'] = room_row.xpath("./TYPE/text()").extract_first()  # 户型
            item_rm['orientation'] = room_row.xpath("./DIRECTION/text()").extract_first()  # 朝向
            item_rm['orientation'] = room_row.xpath("./DIRECTION/text()").extract_first()  # 朝向
            item_rm['buildingStructure'] = room_row.xpath("./STRUCTURE/text()").extract_first()  # 建筑结构
            item_rm['recordTotalPrice'] = room_row.xpath("./SELLTOTALPRICES/text()").extract_first()  # 备案总价
            item_rm['recordUnitPrice'] = room_row.xpath("./SELLUNITPRICE/text()").extract_first()  # 备案单价
            item_rm['floorHeight'] = room_row.xpath("./FLOORHIGH/text()").extract_first()  # 层高
            item_rm['roomFloor'] = room_row.xpath("./FLOORON/text()").extract_first()  # 房屋楼层
            item_rm['roomFloorName'] = room_row.xpath("./FLOORNAME/text()").extract_first()  # 房屋楼层名称
            item_rm['unitName'] = self.get_nuit_name(room_row)  # 单元
            floor_end = room_row.xpath("./FLOORONEND/text()").extract_first()
            item_rm['isDuplex'] = self.is_duplex(item_rm['roomFloor'], floor_end)  # 是否跃层
            item_rm['saleStatus'] = self.get_sale_status(room_row)  # 销售状态
            yield item_rm

            bd_unit_set.add(item_rm['unitName']) if item_rm['unitName'] else False
            bd_floor_set.add(item_rm['roomFloor']) if item_rm['roomFloor'] else False
            bd_floor_set.add(floor_end) if floor_end else False
        # 楼栋单元列表
        item_bd['unitLi'] = list(bd_unit_set) if len(bd_unit_set) else ['1']
        # 楼栋 地上和地下 总楼层
        item_bd['floorOverGroundNo'], item_bd['floorUnderGroundNo'] = self.get_total_floor(bd_floor_set)
        yield item_bd

    @staticmethod
    def get_total_page_num(cont: str):
        """
        提取总页数
        :param cont:
        :return:
        """
        regex = re.compile(r'\d+')
        try:
            total_num = int(regex.findall(cont)[0])
            total_page_num = total_num // 15 + 1 if total_num % 15 else total_num // 15
        except:
            logger.warning('预售许可证列表总页数提取出错')
            total_page_num = 177  # 设置总页数默认值为177
        return total_page_num

    def create_room_li_url(self, resp_str):
        """
        构造房号列表url
        :param resp:
        :return:
        """
        try:
            sid = random.random()
            client_showMode = re.findall(r"client_showMode=\'(.*)\'", resp_str)[0]
            client_stanID = re.findall(r"client_stanID=\'(.*)\'", resp_str)[0]
            client_realtypeID = re.findall(r"client_realtypeID=\'(.*)\'", resp_str)[0]
            client_mainTable = re.findall(r"client_mainTable=\'(.*)\'", resp_str)[0]
            client_mainno = re.findall(r"client_mainno=\'(.*)\'", resp_str)[0]
            client_buildID = re.findall(r"client_buildID=\'(.*)\'", resp_str)[0]
            params_url = "client_showMode=" + client_showMode + "&client_stanID=" + client_stanID + "&client_realtypeID=" + client_realtypeID + "&client_mainTable=" + client_mainTable + "&client_mainno=" + client_mainno + "&client_buildID=" + client_buildID + "&floorStart=-100&floorEnd=-100&roomNoStart=-100&roomNoEnd=-100&cancelBldroomShow=1&client_showiscansale=0&client_showRoomCond=&pmBldRoomID=undefined"
        except Exception as e:
            raise Exception(e)
        else:
            return self.room_li_url.format(sid, params_url)

    @staticmethod
    def get_nuit_name(room_row):
        """
        获取单元
        :param room_row:
        :return:
        """
        try:
            unit_name = room_row.xpath("./BUILDUNIT/text()").extract_first()
            assert unit_name and unit_name.strip() and '无' not in unit_name
        except:
            return
        else:
            return unit_name.strip()

    @staticmethod
    def is_duplex(floor_on, floor_end):
        """
        判断是否跃层
        :param floor_on:
        :param floor_end:
        :return:
        """
        try:
            floor_on = int(floor_on)
            floor_end = int(floor_end)
            ret = True if floor_end > floor_on else False
        except:
            return
        else:
            return ret

    @staticmethod
    def get_sale_status(room_row):
        """
        获取销售状态
        :return:
        """
        base_node_li = ['isrestrict', 'isMortgageForeshow', 'ismortgageregist', 'isHaveRegistForeshow', 'ishaveregist',
                        'ishavesign', 'issubscribe', 'ismortgage', 'ishighmortgage', 'iscansale', 'ismghavesign',
                        'ismghaveregist', 'ismthavesign', 'ismthaveregist', 'ishmthavesign', 'ishmthaveregist']
        base_status_li = ['已限制登记', '已预购商品房抵押权预告登记', '已预购商品房抵押登记', '已预购商品房预告登记', '已预售登记', '已期房签约', '已期房认购', '已在建工程抵押',
                          '已在建工程最高额抵押', '已预售许可', '已预购商品房抵押合同网签', '已预购商品房抵押合同备案', '已在建工程抵押合同网签', '已在建工程抵押合同备案',
                          '已在建工程最高额抵押合同网签', '已在建工程最高额抵押合同备案']
        for index, node_str in enumerate(base_node_li):
            node_value = room_row.xpath(f"./{node_str.upper()}/text()").extract_first()
            try:
                assert int(node_value) == -1
            except:
                continue
            else:
                return base_status_li[index]
        return '不可租售押'

    @staticmethod
    def get_total_floor(floor_set):
        """
        获取楼栋 地上和地下 总楼层
        :param floor_set:
        :return:
        """
        regex = re.compile(r'-?\d+')
        over_li = list()
        under_li = list()
        try:
            floor_li = [int(regex.findall(i)[0]) for i in list(floor_set) if regex.findall(i)]
            for i in floor_li:
                over_li.append(i) if i > 0 else under_li.append(i)
            over_li = sorted(over_li, reverse=True)
            under_li = sorted(under_li, reverse=False)
            floor_over = over_li[0] if over_li else None
            floor_under = abs(under_li[0]) if under_li else None
        except:
            logger.error(f'提取楼栋总楼层出错，msg:{floor_set}')
            return None, None
        else:
            return floor_over, floor_under
